package com.samp.solr.file2solr;

import com.samp.util.MapUtils;
import org.apache.http.client.HttpClient;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;

import java.io.BufferedReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;


public class FileToSolr {

    private static final String SOLR_FIELD_ID = "id";
    private static final String SOLR_FIELD_CONTENT = "content";
    private static final String SOLR_FIELD_INTENTION = "intention";
    private static final String SOLR_FIELD_CONTENT_FULL_MATCH = "contentFullMatch";

//    private static final String BASE_SOLR_URL = "http://localhost:8080/solr/gcc-omcp";
    private static final String BASE_SOLR_URL = "http://localhost:8080/solr/test";

//    private static int size = 40000000;
//    private static BloomFilter<String> bloomFilter =
//                            BloomFilter.create(Funnels.stringFunnel(Charset.forName("utf-8")), size, 0.0000001);


    public static void addFile(String rawPath, boolean delete) throws Exception{
        Path filePath = Paths.get(rawPath);
        BufferedReader bufferReader = Files.newBufferedReader(filePath, StandardCharsets.UTF_8);
        String line;
        try {
//            SolrClient solrCliect =new HttpSolrClient.Builder(BASE_SOLR_URL).build();
            SolrClient solrCliect = getUpdateSolrClient();
            //删除原有数据
            if( delete ) {
                solrCliect.deleteByQuery(SOLR_FIELD_CONTENT+":*");
            }
            int i = 0 ;
            long start = System.currentTimeMillis();
            while( (line = bufferReader.readLine()) != null ){
//                String transText = DataAnalyze.transferText(line);
//                if( bloomFilter.mightContain(transText) ){
//                    continue;
//                }
//                bloomFilter.put(transText);
                SolrInputDocument document = new SolrInputDocument();
                String[] words = line.split("\t");
                if( words.length == 3){
                    document.addField(SOLR_FIELD_ID, UUID.randomUUID().toString().toUpperCase());
                    document.addField(SOLR_FIELD_CONTENT, words[0]);
                    document.addField(SOLR_FIELD_CONTENT_FULL_MATCH, words[0]);
                    document.addField(SOLR_FIELD_INTENTION, words[1]);
                    solrCliect.add(document);
                }else {
                    System.out.println("error line:"+line);
                    continue;
                }
                i++;
                if( i % 50000 == 0  ){
//                    solrCliect.commit();
                    System.out.println("commit doc every 50000 docs.");
                    System.out.println(".......cost:"+(System.currentTimeMillis()-start));
                    start = System.currentTimeMillis();
                }
            }
            solrCliect.commit();
            System.out.println("last commit.");
        } catch (IOException e) {
            e.printStackTrace();
        }finally{
            bufferReader.close();
        }
    }

    private static SolrClient getUpdateSolrClient() throws Exception{
        ModifiableSolrParams params = new ModifiableSolrParams();
        params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 128);
        params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 32);
        params.set(HttpClientUtil.PROP_FOLLOW_REDIRECTS, false);
        params.set(HttpClientUtil.PROP_ALLOW_COMPRESSION, true);
        params.set(HttpClientUtil.PROP_USE_RETRY, true);
        HttpClient httpClient = HttpClientUtil.createClient(params);
        ConcurrentUpdateSolrClient updateSolrClient = new ConcurrentUpdateSolrClient.Builder(BASE_SOLR_URL)
                .withHttpClient(httpClient).withQueueSize(100).withThreadCount(10).build();
        updateSolrClient.setConnectionTimeout(1000);
//        updateSolrClient.setSoTimeout(1000);
        updateSolrClient.setPollQueueTime(500);
        return updateSolrClient;
    }

    /**
     * 统计命名实体个数，比如一期，三期，九月份。
     */
    private static void statisticsNameEntry(String rawPath) throws Exception{
        Path filePath = Paths.get(rawPath);
        BufferedReader bufferReader = Files.newBufferedReader(filePath, StandardCharsets.UTF_8);
        String line;
        try {
            Set<String> neSet = new HashSet<>();
            while( (line = bufferReader.readLine()) != null ){
                String[] words = line.split("\t");
                if( words.length == 3){
                    String word = words[0];
                    String[] splits = word.split(" ");
                    for( String s: splits ){
                        if( isNameEntry(s) && !neSet.contains(s)){
                            neSet.add(s);
                        }
                    }
                }else {
                    System.out.println("error line:"+line);
                    continue;
                }
            }
            System.out.println(neSet);
        }catch (Exception e) {
            e.printStackTrace();
        }finally{
            bufferReader.close();
        }
    }

    private static boolean isNameEntry(String word){
        if( word.indexOf("期") > -1 ){
            return true;
        }
        return false;
    }

    /**
     * 统计结果集中，每个意图对应语料的比例。
     * @param rawPath
     * @throws Exception
     */
    private static void statisticsWordsInIntentions(String rawPath) throws Exception{
        Path filePath = Paths.get(rawPath);
        BufferedReader bufferReader = Files.newBufferedReader(filePath, StandardCharsets.UTF_8);
        String line;
        try {
            int count = 0;
            Map<String, List<String>> dataMap = new HashMap<>();
            while( (line = bufferReader.readLine()) != null ){
                String[] words = line.split("\t");
                if( words.length == 3){
                    if( dataMap.get(words[1]) == null ){
                        dataMap.put(words[1],new ArrayList<>());
                    }
                    dataMap.get(words[1]).add(words[0]);
                }else {
                    System.out.println("error line:"+line);
                    continue;
                }
                count++;
            }
            Map<String,Integer> sizeMap = new HashMap<>();
            Set<String> keySet = dataMap.keySet();
            for(String str: keySet){
                sizeMap.put(str, dataMap.get(str).size());
            }
            LinkedHashMap<String,Integer> sortedMap = (LinkedHashMap)MapUtils.sortedByValue(sizeMap);
            Iterator iter = sortedMap.entrySet().iterator();
            int countDown = 3;
            while (iter.hasNext()) {
                Map.Entry entry = (Map.Entry) iter.next();
                Object key = entry.getKey();
                Object val = entry.getValue();
                System.out.println(key+":"+val);
                if( countDown > 0 ){
                    String saveFilePath = "D:\\gcc-omcp\\temp\\";
                    String saveFileName =key+".txt";
                    ErrorIntentionExtract.saveResultToFile(dataMap.get(key), saveFilePath+saveFileName);
                }
                countDown--;
            }

//            System.out.println(sortedMap);
        } catch (Exception e) {
            e.printStackTrace();
        }finally{
            bufferReader.close();
        }
    }

    /**
     * 通过意图增加语料时，先要将所有的意图删除掉，避免重复.
     * @throws Exception
     */
    private static void deleteAllIntentions( String rawPath ) throws Exception{
        Path filePath = Paths.get(rawPath);
        BufferedReader bufferReader = Files.newBufferedReader(filePath, StandardCharsets.UTF_8);
        String line;
        try {
//            SolrClient solrCliect =new HttpSolrClient.Builder(BASE_SOLR_URL).build();
            SolrClient solrCliect = getUpdateSolrClient();
            while( (line = bufferReader.readLine()) != null ){
                String[] words = line.split("\t");
                String deleteQuery = SOLR_FIELD_CONTENT_FULL_MATCH+":"+"\""+words[0]+"\"";
//                System.out.println(deleteQuery);
                solrCliect.deleteByQuery(deleteQuery);

            }
            solrCliect.commit();
            System.out.println("last commit.");
        } catch (IOException e) {
            e.printStackTrace();
        }finally{
            bufferReader.close();
        }
    }

    public static void main(String[] args) {
//        String sourceDir = "D:\\gcc-omcp\\new\\trans\\";
//        Path dir = Paths.get(sourceDir);
//        //通过Files.newDirectoryStream(dir)来遍历文件夹
//        try( DirectoryStream<Path> stream = Files.newDirectoryStream(dir) ){
//            for(Path path : stream){
//                if( !Files.isDirectory(path) ){
//                    System.out.println("process........"+path.getFileName());
//                    String rawPath = sourceDir+path.getFileName();
//                    addFile(rawPath);
//                }
//            }
//        }catch(Exception e){
//            e.printStackTrace();
//        }
        long start = System.currentTimeMillis();
        String rawPath = "D:\\gcc-omcp\\new\\data\\027_total_words.txt";
//        rawPath = "D:\\gcc-omcp\\all_intention_with_words.txt";
        boolean deleteExist = true;
        try {
//            deleteAllIntentions(rawPath);
            addFile(rawPath, deleteExist);
//            statisticsWordsInIntentions(rawPath);
//            statisticsNameEntry(rawPath);
        }catch (Exception e){
            e.printStackTrace();
        }
        System.out.println("总计耗时："+(System.currentTimeMillis()-start)/(1000.0*60.0)+" 分钟");
    }
}
